{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pretty printing has been turned OFF\n",
      "Pretty printing has been turned ON\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pprint\n",
    "from Environment import GridworldEnv\n",
    "from pprint import PrettyPrinter\n",
    "\n",
    "%pprint\n",
    "pp = PrettyPrinter(indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.25  0.25  0.25  0.25]]\n"
     ]
    }
   ],
   "source": [
    "env = GridworldEnv()\n",
    "random_policy = np.ones([env.nS, env.nA])/env.nA\n",
    "print(random_policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def policy_eval(policy, environment, discount_factor=1.0, theta=1.0):\n",
    "    env = environment # 环境变量\n",
    "    \n",
    "    # 初始化一个全0的价值函数\n",
    "    V = np.zeros(env.nS)\n",
    "    \n",
    "    # 迭代开始\n",
    "    for _ in range(10000):\n",
    "        delta = 0\n",
    "        \n",
    "        # 对于GridWorld中的每一个状态都进行全备份\n",
    "        for s in range(env.nS):\n",
    "            v = 0\n",
    "            # 检查下一个有可能执行的动作\n",
    "            for a, action_prob in enumerate(policy[s]):\n",
    "                \n",
    "                # 对于每一个动作检查下一个状态\n",
    "                for  prob, next_state, reward, done in env.P[s][a]:\n",
    "                    # 累积计算下一个动作的期望价值\n",
    "                    v += action_prob * prob * (reward + discount_factor * V[next_state])\n",
    "                    \n",
    "            # 选出最大的变化量\n",
    "            delta = max(delta, np.abs(v - V[s]))\n",
    "            V[s] = v\n",
    "        \n",
    "        print(\"=\"*60, _)\n",
    "        print(V.reshape(env.shape))\n",
    "        \n",
    "        # 停止标志位\n",
    "        if delta <= theta:\n",
    "            break\n",
    "    \n",
    "    return np.array(V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================ 0\n",
      "[[-1.         -1.25       -1.3125     -1.328125  ]\n",
      " [-1.25       -1.         -1.578125   -1.        ]\n",
      " [-1.3125     -1.578125   -1.7890625  -1.        ]\n",
      " [-1.         -1.64453125 -1.85839844  1.        ]]\n",
      "============================================================ 1\n",
      "[[-2.125      -2.421875   -2.66015625 -2.57910156]\n",
      " [-2.421875   -2.         -2.86230469 -2.        ]\n",
      " [-2.578125   -3.00292969 -3.1809082  -2.        ]\n",
      " [-2.         -3.12646484 -2.79144287  2.        ]]\n",
      "============================================================ 2\n",
      "[[-3.2734375  -3.58886719 -3.92260742 -3.77020264]\n",
      " [-3.56835938 -3.         -4.02587891 -3.        ]\n",
      " [-3.78735352 -4.27368164 -4.27275085 -3.        ]\n",
      " [-3.         -4.29789734 -3.34052277  3.        ]]\n",
      "============================================================ 3\n",
      "[[-4.42602539 -4.734375   -5.11326599 -4.91341782]\n",
      " [-4.69543457 -4.         -5.09650421 -4.        ]\n",
      " [-4.93911743 -5.37744141 -5.2036171  -4.        ]\n",
      " [-4.         -5.25396538 -3.69952631  4.        ]]\n",
      "============================================================ 4\n",
      "[[-5.57046509 -5.85452652 -6.24442863 -6.01781607]\n",
      " [-5.80125427 -5.         -6.11201143 -5.        ]\n",
      " [-6.02945328 -6.37175894 -6.04582417 -5.        ]\n",
      " [-5.         -6.08131266 -3.95666578  5.        ]]\n",
      "============================================================ 5\n",
      "[[-6.69917774 -6.94953322 -7.33094734 -7.09164487]\n",
      " [-6.88247132 -6.         -7.09419288 -6.        ]\n",
      " [-7.07092088 -7.29951443 -6.83759327 -6.        ]\n",
      " [-6.         -6.83437322 -4.15715807  6.        ]]\n",
      "============================================================ 6\n",
      "[[-7.80759001 -8.02201764 -8.38470068 -8.1419976 ]\n",
      " [-7.94024555 -7.         -8.05557349 -7.        ]\n",
      " [-8.07767022 -8.18740918 -7.60003518 -7.        ]\n",
      " [-7.         -7.54473512 -4.32548209  7.        ]]\n",
      "============================================================ 7\n",
      "[[-8.8943608  -9.07526978 -9.41438539 -9.17459515]\n",
      " [-8.97806914 -8.         -9.00360514 -8.        ]\n",
      " [-9.06078713 -9.05138936 -8.34511915 -8.        ]\n",
      " [-8.         -8.23040164 -4.47525072  8.        ]]\n",
      "============================================================ 8\n",
      "[[ -9.96051513 -10.11254258 -10.42628206 -10.19386809]\n",
      " [ -9.99984285  -9.          -9.9428503   -9.        ]\n",
      " [-10.02800484  -9.90088141  -9.07974561  -9.        ]\n",
      " [ -9.          -8.90163344  -4.61415744   9.        ]]\n",
      "============================================================ 9\n",
      "[[-11.00835392 -11.13679464 -11.42494877 -11.20317124]\n",
      " [-11.0090504  -10.         -10.8761736  -10.        ]\n",
      " [-10.98448416 -10.7414658   -9.80794921 -10.        ]\n",
      " [-10.          -9.56431417  -4.74660521  10.        ]]\n",
      "============================================================ 10\n",
      "[[-12.04063822 -12.15059541 -12.41372225 -12.20501618]\n",
      " [-12.0085432  -11.         -11.80541787 -11.        ]\n",
      " [-11.93362329 -11.57647167 -10.53212369 -11.        ]\n",
      " [-11.         -10.22184776  -4.87514416  11.        ]]\n",
      "============================================================ 11\n",
      "[[-13.06010376 -13.15610536 -13.39506542 -13.20127445]\n",
      " [-13.00056756 -12.         -12.73179728 -12.        ]\n",
      " [-12.87766563 -12.40790927 -11.25371268 -12.        ]\n",
      " [-12.         -10.8762253   -5.00127053  12.        ]]\n",
      "============================================================ 12\n",
      "[[-14.06922011 -14.15509772 -14.37080871 -14.1933394 ]\n",
      " [-13.98686333 -13.         -13.65613035 -13.        ]\n",
      " [-13.81810956 -13.23701188 -11.97360319 -13.        ]\n",
      " [-13.         -11.52862693  -5.12587516  13.        ]]\n",
      "============================================================ 13\n",
      "[[-15.07010032 -15.14900169 -15.34232004 -15.18224971]\n",
      " [-14.9687683  -14.         -14.57898081 -14.        ]\n",
      " [-14.75597243 -14.06455064 -12.69235165 -14.        ]\n",
      " [-14.         -12.17976318  -5.2494975   14.        ]]\n",
      "============================================================ 14\n",
      "[[-16.06449266 -16.1389536  -16.31062604 -16.16878136]\n",
      " [-15.94730835 -15.         -15.50074442 -15.        ]\n",
      " [-15.69195786 -14.89101817 -13.41031502 -15.        ]\n",
      " [-15.         -12.83006971  -5.37247056  15.        ]]\n",
      "Reshaped Grid Value Function:\n",
      "[[-16.06449266 -16.1389536  -16.31062604 -16.16878136]\n",
      " [-15.94730835 -15.         -15.50074442 -15.        ]\n",
      " [-15.69195786 -14.89101817 -13.41031502 -15.        ]\n",
      " [-15.         -12.83006971  -5.37247056  15.        ]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "v = policy_eval(random_policy, env)\n",
    "print(\"Reshaped Grid Value Function:\")\n",
    "print(v.reshape(env.shape))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([2, 1, 3, 1, 2, 0, 1, 0, 1, 1, 0, 1, 0, 3, 2, 3]),\n",
      " array([0, 3, 1, 1, 0, 3, 3, 1, 2, 1, 0, 1, 2, 1, 2, 1]),\n",
      " array([2, 2, 2, 1, 2, 3, 1, 1, 1, 1, 1, 1, 2, 0, 3, 1]),\n",
      " array([3, 2, 0, 0, 2, 1, 1, 2, 0, 1, 3, 3, 0, 3, 0, 1]),\n",
      " array([1, 1, 1, 1, 1, 1, 3, 1, 0, 3, 1, 0, 0, 2, 3, 2]),\n",
      " array([0, 0, 0, 0, 0, 2, 3, 1, 1, 3, 2, 1, 3, 1, 0, 3]),\n",
      " array([2, 1, 3, 3, 2, 3, 2, 3, 2, 3, 3, 1, 0, 3, 0, 3]),\n",
      " array([2, 2, 2, 1, 3, 1, 0, 1, 3, 0, 2, 3, 3, 1, 0, 3]),\n",
      " array([0, 1, 2, 3, 0, 2, 2, 0, 0, 2, 2, 2, 3, 3, 1, 3]),\n",
      " array([3, 1, 2, 0, 0, 3, 0, 0, 3, 1, 3, 3, 0, 2, 1, 1])]\n"
     ]
    }
   ],
   "source": [
    "def gen_random_policy():\n",
    "    return np.random.choice(4, size=((16)))\n",
    "\n",
    "n_policy = 100\n",
    "policy_pop = [gen_random_policy() for _ in range(n_policy)]\n",
    "pprint.pprint(policy_pop[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test the Spesfiy Input Policy\n",
    "\n",
    "if I input a speciay policy what the value funciton calcuate by the policy evalutaiton."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  1.  0.]\n",
      " [ 0.  1.  0.  0.]\n",
      " [ 0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  1.]\n",
      " [ 0.  0.  1.  0.]\n",
      " [ 1.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.]\n",
      " [ 1.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.]\n",
      " [ 0.  0.  1.  0.]\n",
      " [ 0.  0.  1.  0.]\n",
      " [ 1.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.]\n",
      " [ 0.  1.  0.  0.]\n",
      " [ 1.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "input_policy = [2,1,2,3,2,0,2,0,1,2,2,0,0,1,1,0]\n",
    "\n",
    "env = GridworldEnv()\n",
    "policy = np.zeros([env.nS, env.nA])\n",
    "\n",
    "for _, x in enumerate(input_policy):\n",
    "    policy[_][x] = 1\n",
    "    \n",
    "print(policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def policy_eval(policy, environment, discount_factor=1.0, theta=0.1):\n",
    "    env = environment # 环境变量\n",
    "    \n",
    "    # 初始化一个全0的价值函数\n",
    "    V = np.zeros(env.nS)\n",
    "    \n",
    "    # 迭代开始\n",
    "    for _ in range(50):\n",
    "        delta = 0\n",
    "        \n",
    "        # 对于GridWorld中的每一个状态都进行全备份\n",
    "        for s in range(env.nS):\n",
    "            v = 0\n",
    "            # 检查下一个有可能执行的动作\n",
    "            for a, action_prob in enumerate(policy[s]):\n",
    "                \n",
    "                # 对于每一个动作检查下一个状态\n",
    "                for  prob, next_state, reward, done in env.P[s][a]:\n",
    "                    # 累积计算下一个动作的期望价值\n",
    "                    v += action_prob * prob * (reward + discount_factor * V[next_state])\n",
    "                    \n",
    "            # 选出最大的变化量\n",
    "            delta = max(delta, np.abs(v - V[s]))\n",
    "            V[s] = v\n",
    "        \n",
    "        print(\"=\"*60, _)\n",
    "        print(V.reshape(env.shape))\n",
    "        \n",
    "        # 停止标志位\n",
    "        if delta <= theta:\n",
    "            break\n",
    "    \n",
    "    return np.array(V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================ 0\n",
      "[[-1. -2. -1. -1.]\n",
      " [-1. -1. -1. -1.]\n",
      " [-1. -1. -1. -1.]\n",
      " [-1. -1. -1.  1.]]\n",
      "============================================================ 1\n",
      "[[-2. -3. -2. -2.]\n",
      " [-2. -2. -2. -2.]\n",
      " [-2. -2. -2. -2.]\n",
      " [-2. -2.  0.  2.]]\n",
      "============================================================ 2\n",
      "[[-3. -4. -3. -3.]\n",
      " [-3. -3. -3. -3.]\n",
      " [-3. -3. -1. -3.]\n",
      " [-3. -1.  1.  3.]]\n",
      "============================================================ 3\n",
      "[[-4. -5. -4. -4.]\n",
      " [-4. -4. -2. -4.]\n",
      " [-4. -2.  0. -4.]\n",
      " [-4.  0.  2.  4.]]\n",
      "============================================================ 4\n",
      "[[-5. -6. -3. -5.]\n",
      " [-5. -5. -1. -5.]\n",
      " [-3. -1.  1. -5.]\n",
      " [-5.  1.  3.  5.]]\n",
      "============================================================ 5\n",
      "[[-6. -7. -2. -6.]\n",
      " [-4. -6.  0. -6.]\n",
      " [-2.  0.  2. -6.]\n",
      " [-6.  2.  4.  6.]]\n",
      "============================================================ 6\n",
      "[[-5. -6. -1. -7.]\n",
      " [-3. -7.  1. -7.]\n",
      " [-1.  1.  3. -7.]\n",
      " [-7.  3.  5.  7.]]\n",
      "============================================================ 7\n",
      "[[-4. -5.  0. -8.]\n",
      " [-2. -8.  2. -8.]\n",
      " [ 0.  2.  4. -8.]\n",
      " [-8.  4.  6.  8.]]\n",
      "============================================================ 8\n",
      "[[-3. -4.  1. -9.]\n",
      " [-1. -9.  3. -9.]\n",
      " [ 1.  3.  5. -9.]\n",
      " [-9.  5.  7.  9.]]\n",
      "============================================================ 9\n",
      "[[ -2.  -3.   2. -10.]\n",
      " [  0. -10.   4. -10.]\n",
      " [  2.   4.   6. -10.]\n",
      " [-10.   6.   8.  10.]]\n",
      "============================================================ 10\n",
      "[[ -1.  -2.   3. -11.]\n",
      " [  1. -11.   5. -11.]\n",
      " [  3.   5.   7. -11.]\n",
      " [-11.   7.   9.  11.]]\n",
      "============================================================ 11\n",
      "[[  0.  -1.   4. -12.]\n",
      " [  2. -12.   6. -12.]\n",
      " [  4.   6.   8. -12.]\n",
      " [-12.   8.  10.  12.]]\n",
      "============================================================ 12\n",
      "[[  1.   0.   5. -13.]\n",
      " [  3. -13.   7. -13.]\n",
      " [  5.   7.   9. -13.]\n",
      " [-13.   9.  11.  13.]]\n",
      "============================================================ 13\n",
      "[[  2.   1.   6. -14.]\n",
      " [  4. -14.   8. -14.]\n",
      " [  6.   8.  10. -14.]\n",
      " [-14.  10.  12.  14.]]\n",
      "============================================================ 14\n",
      "[[  3.   2.   7. -15.]\n",
      " [  5. -15.   9. -15.]\n",
      " [  7.   9.  11. -15.]\n",
      " [-15.  11.  13.  15.]]\n",
      "============================================================ 15\n",
      "[[  4.   3.   8. -16.]\n",
      " [  6. -16.  10. -16.]\n",
      " [  8.  10.  12. -16.]\n",
      " [-16.  12.  14.  16.]]\n",
      "============================================================ 16\n",
      "[[  5.   4.   9. -17.]\n",
      " [  7. -17.  11. -17.]\n",
      " [  9.  11.  13. -17.]\n",
      " [-17.  13.  15.  17.]]\n",
      "============================================================ 17\n",
      "[[  6.   5.  10. -18.]\n",
      " [  8. -18.  12. -18.]\n",
      " [ 10.  12.  14. -18.]\n",
      " [-18.  14.  16.  18.]]\n",
      "============================================================ 18\n",
      "[[  7.   6.  11. -19.]\n",
      " [  9. -19.  13. -19.]\n",
      " [ 11.  13.  15. -19.]\n",
      " [-19.  15.  17.  19.]]\n",
      "============================================================ 19\n",
      "[[  8.   7.  12. -20.]\n",
      " [ 10. -20.  14. -20.]\n",
      " [ 12.  14.  16. -20.]\n",
      " [-20.  16.  18.  20.]]\n",
      "============================================================ 20\n",
      "[[  9.   8.  13. -21.]\n",
      " [ 11. -21.  15. -21.]\n",
      " [ 13.  15.  17. -21.]\n",
      " [-21.  17.  19.  21.]]\n",
      "============================================================ 21\n",
      "[[ 10.   9.  14. -22.]\n",
      " [ 12. -22.  16. -22.]\n",
      " [ 14.  16.  18. -22.]\n",
      " [-22.  18.  20.  22.]]\n",
      "============================================================ 22\n",
      "[[ 11.  10.  15. -23.]\n",
      " [ 13. -23.  17. -23.]\n",
      " [ 15.  17.  19. -23.]\n",
      " [-23.  19.  21.  23.]]\n",
      "============================================================ 23\n",
      "[[ 12.  11.  16. -24.]\n",
      " [ 14. -24.  18. -24.]\n",
      " [ 16.  18.  20. -24.]\n",
      " [-24.  20.  22.  24.]]\n",
      "============================================================ 24\n",
      "[[ 13.  12.  17. -25.]\n",
      " [ 15. -25.  19. -25.]\n",
      " [ 17.  19.  21. -25.]\n",
      " [-25.  21.  23.  25.]]\n",
      "============================================================ 25\n",
      "[[ 14.  13.  18. -26.]\n",
      " [ 16. -26.  20. -26.]\n",
      " [ 18.  20.  22. -26.]\n",
      " [-26.  22.  24.  26.]]\n",
      "============================================================ 26\n",
      "[[ 15.  14.  19. -27.]\n",
      " [ 17. -27.  21. -27.]\n",
      " [ 19.  21.  23. -27.]\n",
      " [-27.  23.  25.  27.]]\n",
      "============================================================ 27\n",
      "[[ 16.  15.  20. -28.]\n",
      " [ 18. -28.  22. -28.]\n",
      " [ 20.  22.  24. -28.]\n",
      " [-28.  24.  26.  28.]]\n",
      "============================================================ 28\n",
      "[[ 17.  16.  21. -29.]\n",
      " [ 19. -29.  23. -29.]\n",
      " [ 21.  23.  25. -29.]\n",
      " [-29.  25.  27.  29.]]\n",
      "============================================================ 29\n",
      "[[ 18.  17.  22. -30.]\n",
      " [ 20. -30.  24. -30.]\n",
      " [ 22.  24.  26. -30.]\n",
      " [-30.  26.  28.  30.]]\n",
      "============================================================ 30\n",
      "[[ 19.  18.  23. -31.]\n",
      " [ 21. -31.  25. -31.]\n",
      " [ 23.  25.  27. -31.]\n",
      " [-31.  27.  29.  31.]]\n",
      "============================================================ 31\n",
      "[[ 20.  19.  24. -32.]\n",
      " [ 22. -32.  26. -32.]\n",
      " [ 24.  26.  28. -32.]\n",
      " [-32.  28.  30.  32.]]\n",
      "============================================================ 32\n",
      "[[ 21.  20.  25. -33.]\n",
      " [ 23. -33.  27. -33.]\n",
      " [ 25.  27.  29. -33.]\n",
      " [-33.  29.  31.  33.]]\n",
      "============================================================ 33\n",
      "[[ 22.  21.  26. -34.]\n",
      " [ 24. -34.  28. -34.]\n",
      " [ 26.  28.  30. -34.]\n",
      " [-34.  30.  32.  34.]]\n",
      "============================================================ 34\n",
      "[[ 23.  22.  27. -35.]\n",
      " [ 25. -35.  29. -35.]\n",
      " [ 27.  29.  31. -35.]\n",
      " [-35.  31.  33.  35.]]\n",
      "============================================================ 35\n",
      "[[ 24.  23.  28. -36.]\n",
      " [ 26. -36.  30. -36.]\n",
      " [ 28.  30.  32. -36.]\n",
      " [-36.  32.  34.  36.]]\n",
      "============================================================ 36\n",
      "[[ 25.  24.  29. -37.]\n",
      " [ 27. -37.  31. -37.]\n",
      " [ 29.  31.  33. -37.]\n",
      " [-37.  33.  35.  37.]]\n",
      "============================================================ 37\n",
      "[[ 26.  25.  30. -38.]\n",
      " [ 28. -38.  32. -38.]\n",
      " [ 30.  32.  34. -38.]\n",
      " [-38.  34.  36.  38.]]\n",
      "============================================================ 38\n",
      "[[ 27.  26.  31. -39.]\n",
      " [ 29. -39.  33. -39.]\n",
      " [ 31.  33.  35. -39.]\n",
      " [-39.  35.  37.  39.]]\n",
      "============================================================ 39\n",
      "[[ 28.  27.  32. -40.]\n",
      " [ 30. -40.  34. -40.]\n",
      " [ 32.  34.  36. -40.]\n",
      " [-40.  36.  38.  40.]]\n",
      "============================================================ 40\n",
      "[[ 29.  28.  33. -41.]\n",
      " [ 31. -41.  35. -41.]\n",
      " [ 33.  35.  37. -41.]\n",
      " [-41.  37.  39.  41.]]\n",
      "============================================================ 41\n",
      "[[ 30.  29.  34. -42.]\n",
      " [ 32. -42.  36. -42.]\n",
      " [ 34.  36.  38. -42.]\n",
      " [-42.  38.  40.  42.]]\n",
      "============================================================ 42\n",
      "[[ 31.  30.  35. -43.]\n",
      " [ 33. -43.  37. -43.]\n",
      " [ 35.  37.  39. -43.]\n",
      " [-43.  39.  41.  43.]]\n",
      "============================================================ 43\n",
      "[[ 32.  31.  36. -44.]\n",
      " [ 34. -44.  38. -44.]\n",
      " [ 36.  38.  40. -44.]\n",
      " [-44.  40.  42.  44.]]\n",
      "============================================================ 44\n",
      "[[ 33.  32.  37. -45.]\n",
      " [ 35. -45.  39. -45.]\n",
      " [ 37.  39.  41. -45.]\n",
      " [-45.  41.  43.  45.]]\n",
      "============================================================ 45\n",
      "[[ 34.  33.  38. -46.]\n",
      " [ 36. -46.  40. -46.]\n",
      " [ 38.  40.  42. -46.]\n",
      " [-46.  42.  44.  46.]]\n",
      "============================================================ 46\n",
      "[[ 35.  34.  39. -47.]\n",
      " [ 37. -47.  41. -47.]\n",
      " [ 39.  41.  43. -47.]\n",
      " [-47.  43.  45.  47.]]\n",
      "============================================================ 47\n",
      "[[ 36.  35.  40. -48.]\n",
      " [ 38. -48.  42. -48.]\n",
      " [ 40.  42.  44. -48.]\n",
      " [-48.  44.  46.  48.]]\n",
      "============================================================ 48\n",
      "[[ 37.  36.  41. -49.]\n",
      " [ 39. -49.  43. -49.]\n",
      " [ 41.  43.  45. -49.]\n",
      " [-49.  45.  47.  49.]]\n",
      "============================================================ 49\n",
      "[[ 38.  37.  42. -50.]\n",
      " [ 40. -50.  44. -50.]\n",
      " [ 42.  44.  46. -50.]\n",
      " [-50.  46.  48.  50.]]\n",
      "Reshaped Grid Value Function:\n",
      "[[ 38.  37.  42. -50.]\n",
      " [ 40. -50.  44. -50.]\n",
      " [ 42.  44.  46. -50.]\n",
      " [-50.  46.  48.  50.]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "v = policy_eval(policy, env)\n",
    "print(\"Reshaped Grid Value Function:\")\n",
    "print(v.reshape(env.shape))\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
